{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Please input your directory for the top level folder\n",
    "folder name : SUBMISSION MODEL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "dir_ = \"INPUT-PROJECT-DIRECTORY/submission_model/\"  # input only here"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### setting other directory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "raw_data_dir = dir_ + \"2. data/\"\n",
    "processed_data_dir = dir_ + \"2. data/processed/\"\n",
    "log_dir = dir_ + \"4. logs/\"\n",
    "model_dir = dir_ + \"5. models/\"\n",
    "submission_dir = dir_ + \"6. submissions/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "####################################################################################\n",
    "################## 2-3. nonrecursive model by store & dept #########################\n",
    "####################################################################################"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "cvs = [\"private\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "STORES = [\n",
    "    \"CA_1\",\n",
    "    \"CA_2\",\n",
    "    \"CA_3\",\n",
    "    \"CA_4\",\n",
    "    \"TX_1\",\n",
    "    \"TX_2\",\n",
    "    \"TX_3\",\n",
    "    \"WI_1\",\n",
    "    \"WI_2\",\n",
    "    \"WI_3\",\n",
    "]\n",
    "DEPTS = [\n",
    "    \"HOBBIES_1\",\n",
    "    \"HOBBIES_2\",\n",
    "    \"HOUSEHOLD_1\",\n",
    "    \"HOUSEHOLD_2\",\n",
    "    \"FOODS_1\",\n",
    "    \"FOODS_2\",\n",
    "    \"FOODS_3\",\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datetime import datetime, timedelta\n",
    "import gc\n",
    "import numpy as np, pandas as pd\n",
    "import lightgbm as lgb\n",
    "\n",
    "import os, sys, gc, time, warnings, pickle, psutil, random\n",
    "\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def reduce_mem_usage(df, verbose=False):\n",
    "    numerics = [\"int16\", \"int32\", \"int64\", \"float16\", \"float32\", \"float64\"]\n",
    "    start_mem = df.memory_usage().sum() / 1024**2\n",
    "    for col in df.columns:\n",
    "        col_type = df[col].dtypes\n",
    "        if col_type in numerics:\n",
    "            c_min = df[col].min()\n",
    "            c_max = df[col].max()\n",
    "            if str(col_type)[:3] == \"int\":\n",
    "                if (\n",
    "                    c_min > np.iinfo(np.int8).min\n",
    "                    and c_max < np.iinfo(np.int8).max\n",
    "                ):\n",
    "                    df[col] = df[col].astype(np.int8)\n",
    "                elif (\n",
    "                    c_min > np.iinfo(np.int16).min\n",
    "                    and c_max < np.iinfo(np.int16).max\n",
    "                ):\n",
    "                    df[col] = df[col].astype(np.int16)\n",
    "                elif (\n",
    "                    c_min > np.iinfo(np.int32).min\n",
    "                    and c_max < np.iinfo(np.int32).max\n",
    "                ):\n",
    "                    df[col] = df[col].astype(np.int32)\n",
    "                elif (\n",
    "                    c_min > np.iinfo(np.int64).min\n",
    "                    and c_max < np.iinfo(np.int64).max\n",
    "                ):\n",
    "                    df[col] = df[col].astype(np.int64)\n",
    "            else:\n",
    "                if (\n",
    "                    c_min > np.finfo(np.float16).min\n",
    "                    and c_max < np.finfo(np.float16).max\n",
    "                ):\n",
    "                    df[col] = df[col].astype(np.float16)\n",
    "                elif (\n",
    "                    c_min > np.finfo(np.float32).min\n",
    "                    and c_max < np.finfo(np.float32).max\n",
    "                ):\n",
    "                    df[col] = df[col].astype(np.float32)\n",
    "                else:\n",
    "                    df[col] = df[col].astype(np.float64)\n",
    "    end_mem = df.memory_usage().sum() / 1024**2\n",
    "    if verbose:\n",
    "        print(\n",
    "            \"Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)\".format(\n",
    "                end_mem, 100 * (start_mem - end_mem) / start_mem\n",
    "            )\n",
    "        )\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
    "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a"
   },
   "outputs": [],
   "source": [
    "FIRST_DAY = 710\n",
    "remove_feature = [\n",
    "    \"id\",\n",
    "    \"state_id\",\n",
    "    \"store_id\",\n",
    "    #                   'item_id',\n",
    "    \"dept_id\",\n",
    "    \"cat_id\",\n",
    "    \"date\",\n",
    "    \"wm_yr_wk\",\n",
    "    \"d\",\n",
    "    \"sales\",\n",
    "]\n",
    "\n",
    "cat_var = [\"item_id\", \"dept_id\", \"store_id\", \"cat_id\", \"state_id\"] + [\n",
    "    \"event_name_1\",\n",
    "    \"event_name_2\",\n",
    "    \"event_type_1\",\n",
    "    \"event_type_2\",\n",
    "]\n",
    "cat_var = list(set(cat_var) - set(remove_feature))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "grid2_colnm = [\n",
    "    \"sell_price\",\n",
    "    \"price_max\",\n",
    "    \"price_min\",\n",
    "    \"price_std\",\n",
    "    \"price_mean\",\n",
    "    \"price_norm\",\n",
    "    \"price_nunique\",\n",
    "    \"item_nunique\",\n",
    "    \"price_momentum\",\n",
    "    \"price_momentum_m\",\n",
    "    \"price_momentum_y\",\n",
    "]\n",
    "\n",
    "grid3_colnm = [\n",
    "    \"event_name_1\",\n",
    "    \"event_type_1\",\n",
    "    \"event_name_2\",\n",
    "    \"event_type_2\",\n",
    "    \"snap_CA\",\n",
    "    \"snap_TX\",\n",
    "    \"snap_WI\",\n",
    "    \"tm_d\",\n",
    "    \"tm_w\",\n",
    "    \"tm_m\",\n",
    "    \"tm_y\",\n",
    "    \"tm_wm\",\n",
    "    \"tm_dw\",\n",
    "    \"tm_w_end\",\n",
    "]\n",
    "\n",
    "lag_colnm = [\n",
    "    \"sales_lag_28\",\n",
    "    \"sales_lag_29\",\n",
    "    \"sales_lag_30\",\n",
    "    \"sales_lag_31\",\n",
    "    \"sales_lag_32\",\n",
    "    \"sales_lag_33\",\n",
    "    \"sales_lag_34\",\n",
    "    \"sales_lag_35\",\n",
    "    \"sales_lag_36\",\n",
    "    \"sales_lag_37\",\n",
    "    \"sales_lag_38\",\n",
    "    \"sales_lag_39\",\n",
    "    \"sales_lag_40\",\n",
    "    \"sales_lag_41\",\n",
    "    \"sales_lag_42\",\n",
    "    \"rolling_mean_7\",\n",
    "    \"rolling_std_7\",\n",
    "    \"rolling_mean_14\",\n",
    "    \"rolling_std_14\",\n",
    "    \"rolling_mean_30\",\n",
    "    \"rolling_std_30\",\n",
    "    \"rolling_mean_60\",\n",
    "    \"rolling_std_60\",\n",
    "    \"rolling_mean_180\",\n",
    "    \"rolling_std_180\",\n",
    "]\n",
    "\n",
    "mean_enc_colnm = [\"enc_item_id_store_id_mean\", \"enc_item_id_store_id_std\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "########################### Make grid\n",
    "#################################################################################\n",
    "def prepare_data(store, state):\n",
    "\n",
    "    grid_1 = pd.read_pickle(processed_data_dir + \"grid_part_1.pkl\")\n",
    "    grid_2 = pd.read_pickle(processed_data_dir + \"grid_part_2.pkl\")[\n",
    "        grid2_colnm\n",
    "    ]\n",
    "    grid_3 = pd.read_pickle(processed_data_dir + \"grid_part_3.pkl\")[\n",
    "        grid3_colnm\n",
    "    ]\n",
    "\n",
    "    grid_df = pd.concat([grid_1, grid_2, grid_3], axis=1)\n",
    "    del grid_1, grid_2, grid_3\n",
    "    gc.collect()\n",
    "\n",
    "    grid_df = grid_df[\n",
    "        (grid_df[\"store_id\"] == store) & (grid_df[\"dept_id\"] == state)\n",
    "    ]\n",
    "    grid_df = grid_df[grid_df[\"d\"] >= FIRST_DAY]\n",
    "\n",
    "    lag = pd.read_pickle(processed_data_dir + \"lags_df_28.pkl\")[lag_colnm]\n",
    "\n",
    "    lag = lag[lag.index.isin(grid_df.index)]\n",
    "\n",
    "    grid_df = pd.concat([grid_df, lag], axis=1)\n",
    "\n",
    "    del lag\n",
    "    gc.collect()\n",
    "\n",
    "    mean_enc = pd.read_pickle(processed_data_dir + \"mean_encoding_df.pkl\")[\n",
    "        mean_enc_colnm\n",
    "    ]\n",
    "    mean_enc = mean_enc[mean_enc.index.isin(grid_df.index)]\n",
    "\n",
    "    grid_df = pd.concat([grid_df, mean_enc], axis=1)\n",
    "    del mean_enc\n",
    "    gc.collect()\n",
    "\n",
    "    grid_df = reduce_mem_usage(grid_df)\n",
    "\n",
    "    return grid_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "validation = {\n",
    "    \"cv1\": [1551, 1610],\n",
    "    \"cv2\": [1829, 1857],\n",
    "    \"cv3\": [1857, 1885],\n",
    "    \"cv4\": [1885, 1913],\n",
    "    \"public\": [1913, 1941],\n",
    "    \"private\": [1941, 1969],\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### cv1 : 2015-04-28 ~ 2015-06-26\n",
    "\n",
    "### cv2 : 2016-02-01 ~ 2016-02-28\n",
    "\n",
    "### cv3 : 2016-02-29 ~ 2016-03-27\n",
    "\n",
    "### cv4 : 2016-03-28 ~ 2016-04-24"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "########################### Model params\n",
    "#################################################################################\n",
    "lgb_params = {\n",
    "    \"boosting_type\": \"gbdt\",\n",
    "    \"objective\": \"tweedie\",\n",
    "    \"tweedie_variance_power\": 1.1,\n",
    "    \"metric\": \"rmse\",\n",
    "    \"subsample\": 0.5,\n",
    "    \"subsample_freq\": 1,\n",
    "    \"learning_rate\": 0.015,\n",
    "    \"num_leaves\": 2**8 - 1,\n",
    "    \"min_data_in_leaf\": 2**8 - 1,\n",
    "    \"feature_fraction\": 0.5,\n",
    "    \"max_bin\": 100,\n",
    "    \"n_estimators\": 3000,\n",
    "    \"boost_from_average\": False,\n",
    "    \"verbose\": -1,\n",
    "    \"seed\": 1995,\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pred_q(quantile):\n",
    "    print(quantile)\n",
    "\n",
    "    rmsse_bycv = dict()\n",
    "\n",
    "    for cv in cvs:\n",
    "        print(\"cv : day\", validation[cv])\n",
    "\n",
    "        pred_list = []\n",
    "        for store in STORES:\n",
    "            for state in DEPTS:\n",
    "\n",
    "                print(store, state, \"start\")\n",
    "                grid_df = prepare_data(store, state)\n",
    "\n",
    "                model_var = grid_df.columns[\n",
    "                    ~grid_df.columns.isin(remove_feature)\n",
    "                ]\n",
    "\n",
    "                tr_mask = (grid_df[\"d\"] <= validation[cv][0]) & (\n",
    "                    grid_df[\"d\"] >= FIRST_DAY\n",
    "                )\n",
    "                vl_mask = (grid_df[\"d\"] > validation[cv][0]) & (\n",
    "                    grid_df[\"d\"] <= validation[cv][1]\n",
    "                )\n",
    "\n",
    "                model_path = (\n",
    "                    model_dir\n",
    "                    + \"non_recur_model_\"\n",
    "                    + store\n",
    "                    + \"_\"\n",
    "                    + state\n",
    "                    + \".bin\"\n",
    "                )\n",
    "                m_lgb = pickle.load(open(model_path, \"rb\"))\n",
    "\n",
    "                indice = grid_df[vl_mask].index.tolist()\n",
    "                print(\"starting to predict\")\n",
    "                prediction = pd.DataFrame(\n",
    "                    {\n",
    "                        \"y_pred\": m_lgb.predict(\n",
    "                            grid_df[vl_mask][model_var], float(quantile)\n",
    "                        )\n",
    "                    }\n",
    "                )\n",
    "                print(\"done predicting\")\n",
    "                prediction.index = indice\n",
    "\n",
    "                del grid_df, m_lgb, tr_mask, vl_mask\n",
    "                gc.collect()\n",
    "\n",
    "                grid_1 = pd.read_pickle(processed_data_dir + \"grid_part_1.pkl\")\n",
    "                pd.concat([grid_1.iloc[indice], prediction], axis=1).pivot(\n",
    "                    index=\"id\", columns=\"d\", values=\"y_pred\"\n",
    "                ).reset_index().set_index(\"id\").to_csv(\n",
    "                    log_dir\n",
    "                    + f\"submission_storeanddept_{store}_{state}_{cv}.csv\"\n",
    "                )\n",
    "\n",
    "                del grid_1\n",
    "                gc.collect()\n",
    "\n",
    "    ########################### Make submissions\n",
    "    #################################################################################\n",
    "\n",
    "    os.chdir(log_dir)\n",
    "\n",
    "    pri = [a for a in os.listdir() if \"storeanddept\" in a]\n",
    "\n",
    "    submission = (\n",
    "        pd.read_csv(raw_data_dir + \"sample_submission.csv\")\n",
    "        .set_index(\"id\")\n",
    "        .iloc[30490:]\n",
    "    )\n",
    "    sub_id = pd.DataFrame({\"id\": submission.index.tolist()})\n",
    "\n",
    "    fcol = [f\"F{i}\" for i in range(1, 29)]\n",
    "\n",
    "    sub_copy = submission.copy()\n",
    "    for file in pri:\n",
    "        temp = pd.read_csv(log_dir + file)\n",
    "        temp.columns = [\"id\"] + fcol\n",
    "        sub_copy += (\n",
    "            sub_id.merge(temp, how=\"left\", on=\"id\").set_index(\"id\").fillna(0)\n",
    "        )\n",
    "    sub_copy.columns = fcol\n",
    "    sub_copy.to_csv(\n",
    "        submission_dir\n",
    "        + f\"before_ensemble/submission_kaggle_nonrecursive_store_dept_{quantile}.csv\"\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.005\n",
      "cv : day [1941, 1969]\n",
      "CA_1 HOBBIES_1 start\n",
      "starting to predict\n",
      "done predicting\n",
      "CA_1 HOUSEHOLD_1 start\n",
      "starting to predict\n",
      "done predicting\n",
      "CA_1 HOUSEHOLD_2 start\n",
      "starting to predict\n",
      "done predicting\n",
      "CA_1 FOODS_1 start\n",
      "starting to predict\n",
      "done predicting\n",
      "CA_1 FOODS_2 start\n",
      "starting to predict\n",
      "done predicting\n",
      "CA_1 FOODS_3 start\n",
      "starting to predict\n",
      "done predicting\n",
      "CA_2 HOBBIES_1 start\n",
      "starting to predict\n",
      "done predicting\n",
      "CA_2 HOBBIES_2 start\n",
      "starting to predict\n",
      "done predicting\n",
      "CA_2 HOUSEHOLD_1 start\n",
      "starting to predict\n",
      "done predicting\n",
      "CA_2 HOUSEHOLD_2 start\n",
      "starting to predict\n",
      "done predicting\n",
      "CA_2 FOODS_1 start\n",
      "starting to predict\n",
      "done predicting\n",
      "CA_2 FOODS_2 start\n",
      "starting to predict\n",
      "done predicting\n",
      "CA_2 FOODS_3 start\n",
      "starting to predict\n",
      "done predicting\n",
      "CA_3 HOBBIES_1 start\n",
      "starting to predict\n",
      "done predicting\n",
      "CA_3 HOBBIES_2 start\n",
      "starting to predict\n",
      "done predicting\n",
      "CA_3 HOUSEHOLD_1 start\n",
      "starting to predict\n",
      "done predicting\n",
      "CA_3 HOUSEHOLD_2 start\n",
      "starting to predict\n",
      "done predicting\n",
      "CA_3 FOODS_1 start\n",
      "starting to predict\n",
      "done predicting\n",
      "CA_3 FOODS_2 start\n",
      "starting to predict\n",
      "done predicting\n",
      "CA_3 FOODS_3 start\n",
      "starting to predict\n",
      "starting to predict\n",
      "done predicting\n",
      "CA_4 FOODS_1 start\n",
      "starting to predict\n",
      "done predicting\n",
      "CA_4 FOODS_2 start\n",
      "starting to predict\n",
      "done predicting\n",
      "CA_4 FOODS_3 start\n",
      "starting to predict\n",
      "done predicting\n",
      "TX_1 HOBBIES_1 start\n",
      "starting to predict\n",
      "done predicting\n",
      "TX_1 HOBBIES_2 start\n",
      "done predicting\n",
      "TX_1 FOODS_1 start\n",
      "starting to predict\n",
      "done predicting\n",
      "TX_1 FOODS_2 start\n",
      "starting to predict\n",
      "done predicting\n",
      "TX_1 FOODS_3 start\n",
      "starting to predict\n",
      "done predicting\n",
      "TX_2 FOODS_1 start\n",
      "starting to predict\n",
      "done predicting\n",
      "TX_2 FOODS_2 start\n",
      "starting to predict\n",
      "done predicting\n",
      "TX_2 FOODS_3 start\n",
      "starting to predict\n",
      "starting to predict\n"
     ]
    }
   ],
   "source": [
    "import concurrent.futures\n",
    "\n",
    "for quantile in [\n",
    "    \"0.005\",\n",
    "    \"0.025\",\n",
    "    \"0.165\",\n",
    "    \"0.250\",\n",
    "    \"0.500\",\n",
    "    \"0.750\",\n",
    "    \"0.835\",\n",
    "    \"0.975\",\n",
    "    \"0.995\",\n",
    "]:\n",
    "    with concurrent.futures.ThreadPoolExecutor() as executor:\n",
    "        executor.submit(pred_q, quantile)"
   ]
  }
 ],
 "metadata": {
  "hide_input": false,
  "kernelspec": {
   "display_name": "conda_mxnet_p36",
   "language": "python",
   "name": "conda_mxnet_p36"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
